{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "using ReinforcementLearningAnIntroduction\n",
    "\n",
    "using StatsBase, Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = ShortCorridorEnv()\n",
    "ns, na = length(get_observation_space(env)), length(get_action_space(env))\n",
    "\n",
    "n_feature = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "run_once (generic function with 1 method)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function run_once(A)\n",
    "    avg_rewards = []\n",
    "    for p in A\n",
    "        agent = Agent(\n",
    "            policy=VBasedPolicy(\n",
    "                learner=TDLearner(\n",
    "                    approximator=LinearQApproximator(\n",
    "                        weights=[-1.47, 1.47],  # init_weights\n",
    "                        feature_func=(s, a) -> a == 1 ? [0, 1] : [1, 0],\n",
    "                        actions=[1, 2]\n",
    "                    ),\n",
    "                    γ=1.0,\n",
    "                    optimizer=Descent(0.1)\n",
    "                ),\n",
    "                mapping= (obs, V) -> rand() < p ? [0, 1] : [1, 0]\n",
    "            ),\n",
    "            trajectory=EpisodicCompactSARTSATrajectory()\n",
    "        )\n",
    "        hook=TotalRewardPerEpisode()\n",
    "        run(agent, env, StopAfterEpisode(1000),hook)\n",
    "\n",
    "        push!(avg_rewards, mean(hook.rewards[end-100:end]))\n",
    "    end\n",
    "    avg_rewards\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32mProgress: 100%|█████████████████████████████████████████| Time: 0:00:00\u001b[39m\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 2400 1600\">\n",
       "<defs>\n",
       "  <clipPath id=\"clip8400\">\n",
       "    <rect x=\"0\" y=\"0\" width=\"2400\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip8400)\" d=\"\n",
       "M0 1600 L2400 1600 L2400 0 L0 0  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip8401\">\n",
       "    <rect x=\"480\" y=\"0\" width=\"1681\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip8400)\" d=\"\n",
       "M168.627 1487.47 L2352.76 1487.47 L2352.76 47.2441 L168.627 47.2441  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip8402\">\n",
       "    <rect x=\"168\" y=\"47\" width=\"2185\" height=\"1441\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polyline clip-path=\"url(#clip8402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  573.858,1487.47 573.858,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1031.75,1487.47 1031.75,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1489.64,1487.47 1489.64,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1947.52,1487.47 1947.52,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  168.627,1468.98 2352.76,1468.98 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  168.627,1072.68 2352.76,1072.68 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  168.627,676.377 2352.76,676.377 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  168.627,280.075 2352.76,280.075 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  168.627,1487.47 2352.76,1487.47 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  168.627,1487.47 168.627,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  573.858,1487.47 573.858,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1031.75,1487.47 1031.75,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1489.64,1487.47 1489.64,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1947.52,1487.47 1947.52,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  168.627,1468.98 194.836,1468.98 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  168.627,1072.68 194.836,1072.68 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  168.627,676.377 194.836,676.377 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  168.627,280.075 194.836,280.075 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip8400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 573.858, 1541.47)\" x=\"573.858\" y=\"1541.47\">0.2</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip8400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1031.75, 1541.47)\" x=\"1031.75\" y=\"1541.47\">0.4</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip8400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1489.64, 1541.47)\" x=\"1489.64\" y=\"1541.47\">0.6</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip8400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1947.52, 1541.47)\" x=\"1947.52\" y=\"1541.47\">0.8</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip8400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 144.627, 1486.48)\" x=\"144.627\" y=\"1486.48\">-80</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip8400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 144.627, 1090.18)\" x=\"144.627\" y=\"1090.18\">-60</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip8400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 144.627, 693.877)\" x=\"144.627\" y=\"693.877\">-40</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip8400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 144.627, 297.575)\" x=\"144.627\" y=\"297.575\">-20</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip8402)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  230.442,1446.71 344.914,690.385 459.386,434.063 573.858,301.067 688.331,231.42 802.803,190.612 917.275,147.725 1031.75,126.969 1146.22,113.824 1260.69,109.959 \n",
       "  1375.16,99.6001 1489.64,88.0053 1604.11,101.052 1718.58,121.357 1833.05,117.591 1947.52,159.712 2062,227.947 2176.47,341.148 2290.94,716.713 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip8400)\" d=\"\n",
       "M1989.93 251.724 L2280.76 251.724 L2280.76 130.764 L1989.93 130.764  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip8400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1989.93,251.724 2280.76,251.724 2280.76,130.764 1989.93,130.764 1989.93,251.724 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8400)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2013.93,191.244 2157.93,191.244 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip8400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 2181.93, 208.744)\" x=\"2181.93\" y=\"208.744\">y1</text>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = 0.05:0.05:0.95\n",
    "plot(A, mean([run_once(A) for _ in 1:10]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "run_once_RL (generic function with 1 method)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "using Random\n",
    "Random.seed!(123)\n",
    "\n",
    "\n",
    "function run_once_RL(α)\n",
    "    agent = Agent(\n",
    "        policy=ReinforcePolicy(\n",
    "            approximator=LinearQApproximator(\n",
    "                weights=[-1.47, 1.47],  # init_weights\n",
    "                feature_func=(s, a) -> a == 1 ? [0, 1] : [1, 0],\n",
    "                actions=[1, 2]\n",
    "                ),\n",
    "            α=α,\n",
    "            γ=1.0\n",
    "        ),\n",
    "        trajectory=EpisodicCompactSARTSATrajectory()\n",
    "    )\n",
    "\n",
    "    hook = TotalRewardPerEpisode()\n",
    "    run(agent,ShortCorridorEnv(),StopAfterEpisode(1000),hook)\n",
    "    hook.rewards\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32mProgress: 100%|█████████████████████████████████████████| Time: 0:00:00\u001b[39m\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 2400 1600\">\n",
       "<defs>\n",
       "  <clipPath id=\"clip8800\">\n",
       "    <rect x=\"0\" y=\"0\" width=\"2400\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip8800)\" d=\"\n",
       "M0 1600 L2400 1600 L2400 0 L0 0  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip8801\">\n",
       "    <rect x=\"480\" y=\"0\" width=\"1681\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip8800)\" d=\"\n",
       "M168.627 1487.47 L2352.76 1487.47 L2352.76 47.2441 L168.627 47.2441  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip8802\">\n",
       "    <rect x=\"168\" y=\"47\" width=\"2185\" height=\"1441\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polyline clip-path=\"url(#clip8802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  228.379,1487.47 228.379,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  744.02,1487.47 744.02,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1259.66,1487.47 1259.66,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1775.3,1487.47 1775.3,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  2290.94,1487.47 2290.94,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  168.627,1269.14 2352.76,1269.14 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  168.627,929.926 2352.76,929.926 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  168.627,590.715 2352.76,590.715 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  168.627,251.505 2352.76,251.505 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  168.627,1487.47 2352.76,1487.47 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  168.627,1487.47 168.627,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  228.379,1487.47 228.379,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  744.02,1487.47 744.02,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1259.66,1487.47 1259.66,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1775.3,1487.47 1775.3,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2290.94,1487.47 2290.94,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  168.627,1269.14 194.836,1269.14 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  168.627,929.926 194.836,929.926 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  168.627,590.715 194.836,590.715 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  168.627,251.505 194.836,251.505 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip8800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 228.379, 1541.47)\" x=\"228.379\" y=\"1541.47\">0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip8800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 744.02, 1541.47)\" x=\"744.02\" y=\"1541.47\">250</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip8800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1259.66, 1541.47)\" x=\"1259.66\" y=\"1541.47\">500</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip8800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1775.3, 1541.47)\" x=\"1775.3\" y=\"1541.47\">750</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip8800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 2290.94, 1541.47)\" x=\"2290.94\" y=\"1541.47\">1000</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip8800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 144.627, 1286.64)\" x=\"144.627\" y=\"1286.64\">-80</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip8800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 144.627, 947.426)\" x=\"144.627\" y=\"947.426\">-60</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip8800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 144.627, 608.215)\" x=\"144.627\" y=\"608.215\">-40</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip8800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 144.627, 269.005)\" x=\"144.627\" y=\"269.005\">-20</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip8802)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  230.442,1266.93 232.504,1369.03 234.567,1358.69 236.63,1446.71 238.692,1154.14 240.755,1138.2 242.817,1219.27 244.88,1428.74 246.942,1175.85 249.005,1313.4 \n",
       "  251.067,1236.23 253.13,1208.08 255.193,1156.01 257.255,1101.91 259.318,999.973 261.38,960.964 263.443,987.422 265.505,1041.19 267.568,1007.61 269.631,1069.85 \n",
       "  271.693,1045.94 273.756,1209.1 275.818,1092.75 277.881,886.507 279.943,913.983 282.006,1195.7 284.068,1037.8 286.131,1029.48 288.194,1077.99 290.256,827.315 \n",
       "  292.319,874.635 294.381,936.71 296.444,827.315 298.506,1021.34 300.569,878.196 302.632,971.14 304.694,1014.9 306.757,902.62 308.819,887.864 310.882,884.641 \n",
       "  312.944,866.324 315.007,879.214 317.069,822.905 319.132,903.128 321.195,759.134 323.257,913.135 325.32,888.034 327.382,904.146 329.445,795.938 331.507,842.41 \n",
       "  333.57,868.19 335.633,812.39 337.695,680.097 339.758,799.669 341.82,852.416 343.883,718.767 345.945,815.443 348.008,886.846 350.07,850.381 352.133,850.551 \n",
       "  354.196,673.992 356.258,715.715 358.321,850.551 360.383,776.264 362.446,900.415 364.508,712.662 366.571,810.015 368.633,803.74 370.696,641.597 372.759,773.38 \n",
       "  374.821,692.818 376.884,661.61 378.946,738.951 381.009,692.987 383.071,747.94 385.134,696.549 387.197,706.386 389.259,702.825 391.322,802.383 393.384,669.582 \n",
       "  395.447,614.63 397.509,691.631 399.572,663.646 401.634,612.595 403.697,721.651 405.76,756.759 407.822,776.264 409.885,728.435 411.947,807.81 414.01,616.495 \n",
       "  416.072,553.063 418.135,601.74 420.198,629.894 422.26,635.491 424.323,684.168 426.385,625.485 428.448,531.693 430.51,769.649 432.573,670.769 434.635,709.439 \n",
       "  436.698,629.385 438.761,690.783 440.823,590.207 442.886,725.721 444.948,593.429 447.011,614.63 449.073,529.318 451.136,594.786 453.199,604.454 455.261,548.653 \n",
       "  457.324,638.035 459.386,638.205 461.449,632.608 463.511,628.029 465.574,685.864 467.636,612.595 469.699,579.182 471.762,744.378 473.824,560.187 475.887,545.77 \n",
       "  477.949,495.228 480.012,594.786 482.074,575.621 484.137,539.495 486.2,522.365 488.262,544.922 490.325,606.658 492.387,554.081 494.45,573.925 496.512,494.21 \n",
       "  498.575,553.742 500.637,566.123 502.7,507.27 504.763,535.255 506.825,550.858 508.888,651.773 510.95,504.047 513.013,581.048 515.075,513.206 517.138,685.694 \n",
       "  519.2,515.75 521.263,586.475 523.326,651.773 525.388,540.512 527.451,596.991 529.513,471.653 531.576,427.216 533.638,578.843 535.701,525.587 537.764,439.767 \n",
       "  539.826,423.824 541.889,481.32 543.951,605.641 546.014,409.238 548.076,560.695 550.139,428.912 552.201,497.941 554.264,572.398 556.327,544.752 558.389,433.322 \n",
       "  560.452,492.684 562.514,428.234 564.577,446.212 566.639,489.8 568.702,454.183 570.765,452.996 572.827,511.34 574.89,504.726 576.952,468.26 579.015,476.232 \n",
       "  581.077,428.234 583.14,622.092 585.202,387.528 587.265,530.166 589.328,417.548 591.39,403.302 593.453,452.657 595.515,470.465 597.578,501.503 599.64,431.286 \n",
       "  601.703,489.461 603.766,502.86 605.828,469.956 607.891,408.39 609.953,459.78 612.016,500.994 614.078,417.718 616.141,366.328 618.203,450.113 620.266,433.661 \n",
       "  622.329,418.736 624.391,389.224 626.454,400.927 628.516,504.386 630.579,410.086 632.641,420.771 634.704,380.405 636.767,431.117 638.829,560.695 640.892,394.821 \n",
       "  642.954,485.221 645.017,421.11 647.079,409.747 649.142,416.531 651.204,418.736 653.267,431.626 655.33,500.655 657.392,475.553 659.455,403.132 661.517,412.969 \n",
       "  663.58,457.236 665.642,508.966 667.705,423.654 669.767,356.151 671.83,471.483 673.893,435.866 675.955,436.035 678.018,346.145 680.08,404.319 682.143,384.136 \n",
       "  684.205,347.841 686.268,479.963 688.331,430.778 690.393,470.805 692.456,445.703 694.518,372.942 696.581,383.967 698.643,418.396 700.706,429.59 702.768,359.374 \n",
       "  704.831,377.522 706.894,362.257 708.956,407.372 711.019,407.711 713.081,422.637 715.144,378.539 717.206,350.894 719.269,419.923 721.332,365.14 723.394,344.618 \n",
       "  725.457,365.14 727.519,428.573 729.582,363.953 731.644,375.995 733.707,393.295 735.769,347.501 737.832,422.128 739.895,387.698 741.957,378.03 744.02,439.428 \n",
       "  746.082,397.026 748.145,304.422 750.207,398.213 752.27,344.618 754.333,363.614 756.395,417.04 758.458,395.839 760.52,379.896 762.583,459.95 764.645,350.554 \n",
       "  766.708,225.386 768.77,398.383 770.833,287.461 772.896,350.215 774.958,318.329 777.021,429.082 779.083,378.539 781.146,352.929 783.208,381.423 785.271,302.386 \n",
       "  787.334,407.881 789.396,333.933 791.459,318.838 793.521,400.927 795.584,298.655 797.646,315.446 799.709,374.978 801.771,318.16 803.834,319.856 805.897,396.857 \n",
       "  807.959,350.894 810.022,339.7 812.084,357.169 814.147,328.675 816.209,415.174 818.272,357.678 820.334,363.444 822.397,349.028 824.46,346.653 826.522,415.683 \n",
       "  828.585,289.496 830.647,345.297 832.71,301.878 834.772,358.865 836.835,314.089 838.898,256.763 840.96,306.287 843.023,281.355 845.085,396.687 847.148,306.966 \n",
       "  849.21,298.486 851.273,386.171 853.335,367.515 855.398,365.48 857.461,308.831 859.523,354.625 861.586,368.872 863.648,364.971 865.711,292.041 867.773,332.576 \n",
       "  869.836,323.757 871.899,309.849 873.961,324.605 876.024,310.019 878.086,268.296 880.149,274.402 882.211,339.021 884.274,317.312 886.336,362.088 888.399,317.312 \n",
       "  890.462,294.585 892.524,319.856 894.587,323.418 896.649,317.821 898.712,341.396 900.774,287.292 902.837,272.027 904.9,294.076 906.962,255.406 909.025,353.777 \n",
       "  911.087,279.49 913.15,268.805 915.212,241.837 917.275,288.479 919.337,275.758 921.4,293.397 923.463,367.684 925.525,349.876 927.588,313.072 929.65,334.272 \n",
       "  931.713,310.358 933.775,245.738 935.838,252.862 937.901,400.418 939.963,278.302 942.026,278.472 944.088,307.305 946.151,329.523 948.213,295.941 950.276,274.91 \n",
       "  952.338,301.03 954.401,286.104 956.464,263.038 958.526,329.523 960.589,269.144 962.651,266.6 964.714,273.214 966.776,313.411 968.839,310.697 970.901,315.785 \n",
       "  972.964,283.73 975.027,352.25 977.089,308.662 979.152,314.768 981.214,361.239 983.277,291.532 985.339,282.543 987.402,286.783 989.465,281.016 991.527,326.81 \n",
       "  993.59,318.16 995.652,293.397 997.715,260.833 999.777,292.719 1001.84,318.669 1003.9,229.287 1005.97,297.638 1008.03,315.107 1010.09,216.227 1012.15,274.741 \n",
       "  1014.22,249.978 1016.28,291.532 1018.34,301.878 1020.4,280.338 1022.47,294.415 1024.53,278.133 1026.59,292.21 1028.65,336.647 1030.72,341.735 1032.78,235.053 \n",
       "  1034.84,274.232 1036.9,255.067 1038.97,286.613 1041.03,261.172 1043.09,262.529 1045.15,275.928 1047.22,222.672 1049.28,267.278 1051.34,242.516 1053.4,278.472 \n",
       "  1055.47,292.38 1057.53,254.049 1059.59,262.699 1061.65,238.954 1063.72,271.518 1065.78,309.001 1067.84,284.408 1069.9,325.622 1071.97,283.391 1074.03,290.684 \n",
       "  1076.09,296.62 1078.15,291.193 1080.22,294.415 1082.28,276.776 1084.34,289.327 1086.4,257.611 1088.47,300.521 1090.53,272.536 1092.59,312.732 1094.66,281.525 \n",
       "  1096.72,242.685 1098.78,253.371 1100.84,263.208 1102.91,279.999 1104.97,198.418 1107.03,245.908 1109.09,229.117 1111.16,319.856 1113.22,223.181 1115.28,232.679 \n",
       "  1117.34,264.056 1119.41,311.884 1121.47,281.525 1123.53,261.681 1125.59,268.635 1127.66,212.665 1129.72,232.509 1131.78,227.082 1133.84,271.179 1135.91,255.236 \n",
       "  1137.97,225.894 1140.03,268.635 1142.09,239.463 1144.16,221.146 1146.22,222.502 1148.28,237.767 1150.34,249.809 1152.41,217.414 1154.47,260.155 1156.53,294.585 \n",
       "  1158.59,225.216 1160.66,285.596 1162.72,256.593 1164.78,287.292 1166.84,224.368 1168.91,204.015 1170.97,285.087 1173.03,270.161 1175.1,248.622 1177.16,221.485 \n",
       "  1179.22,237.936 1181.28,269.992 1183.35,271.009 1185.41,291.532 1187.47,229.117 1189.53,258.119 1191.6,243.703 1193.66,231.152 1195.72,256.932 1197.78,224.538 \n",
       "  1199.85,207.916 1201.91,310.019 1203.97,272.027 1206.03,207.068 1208.1,276.098 1210.16,298.486 1212.22,262.02 1214.28,243.194 1216.35,251.674 1218.41,230.983 \n",
       "  1220.47,197.401 1222.53,236.24 1224.6,170.773 1226.66,236.919 1228.72,218.941 1230.78,259.476 1232.85,217.414 1234.91,191.295 1236.97,200.114 1239.03,224.877 \n",
       "  1241.1,246.417 1243.16,209.612 1245.22,212.496 1247.28,214.531 1249.35,237.428 1251.41,233.696 1253.47,209.952 1255.53,242.516 1257.6,236.41 1259.66,203.676 \n",
       "  1261.72,251.674 1263.79,238.106 1265.85,223.859 1267.91,194.009 1269.97,237.936 1272.04,212.835 1274.1,260.494 1276.16,291.362 1278.22,294.924 1280.29,285.596 \n",
       "  1282.35,247.774 1284.41,205.542 1286.47,185.528 1288.54,275.758 1290.6,205.881 1292.66,224.368 1294.72,242.685 1296.79,206.729 1298.85,235.562 1300.91,236.749 \n",
       "  1302.97,277.454 1305.04,209.443 1307.1,207.747 1309.16,255.236 1311.22,260.494 1313.29,202.489 1315.35,202.998 1317.41,192.821 1319.47,190.447 1321.54,190.108 \n",
       "  1323.6,238.615 1325.66,228.439 1327.72,244.381 1329.79,249.639 1331.85,274.402 1333.91,195.705 1335.97,197.74 1338.04,189.599 1340.1,257.441 1342.16,224.368 \n",
       "  1344.23,230.304 1346.29,210.63 1348.35,220.637 1350.41,219.28 1352.48,227.76 1354.54,213.004 1356.6,228.439 1358.66,227.591 1360.73,206.051 1362.79,237.258 \n",
       "  1364.85,216.905 1366.91,187.733 1368.98,216.736 1371.04,190.277 1373.1,237.597 1375.16,179.762 1377.23,227.76 1379.29,234.205 1381.35,182.306 1383.41,193.5 \n",
       "  1385.48,254.727 1387.54,180.61 1389.6,214.192 1391.66,218.941 1393.73,198.418 1395.79,260.324 1397.85,189.429 1399.91,204.524 1401.98,180.101 1404.04,203.507 \n",
       "  1406.1,197.401 1408.16,231.831 1410.23,211.648 1412.29,257.95 1414.35,164.837 1416.41,229.117 1418.48,208.086 1420.54,226.234 1422.6,158.9 1424.67,241.498 \n",
       "  1426.73,201.471 1428.79,190.617 1430.85,196.383 1432.92,175.691 1434.98,187.903 1437.04,187.733 1439.1,228.099 1441.17,292.719 1443.23,210.969 1445.29,169.246 \n",
       "  1447.35,163.14 1449.42,147.876 1451.48,218.262 1453.54,195.366 1455.6,235.223 1457.67,215.04 1459.73,178.744 1461.79,184.341 1463.85,203.676 1465.92,190.447 \n",
       "  1467.98,168.229 1470.04,201.471 1472.1,245.738 1474.17,177.048 1476.23,212.326 1478.29,190.108 1480.35,199.775 1482.42,190.447 1484.48,224.707 1486.54,156.695 \n",
       "  1488.6,220.976 1490.67,214.022 1492.73,187.055 1494.79,165.685 1496.85,193.669 1498.92,190.617 1500.98,211.478 1503.04,205.711 1505.1,217.923 1507.17,193.33 \n",
       "  1509.23,181.797 1511.29,215.718 1513.36,197.91 1515.42,152.116 1517.48,186.207 1519.54,196.214 1521.61,181.627 1523.67,216.905 1525.73,206.051 1527.79,181.119 \n",
       "  1529.86,209.273 1531.92,191.125 1533.98,155.339 1536.04,153.643 1538.11,196.383 1540.17,181.288 1542.23,162.971 1544.29,181.797 1546.36,161.614 1548.42,182.475 \n",
       "  1550.48,178.575 1552.54,234.714 1554.61,211.139 1556.67,189.09 1558.73,177.896 1560.79,188.751 1562.86,227.76 1564.92,169.755 1566.98,191.295 1569.04,179.762 \n",
       "  1571.11,158.731 1573.17,175.182 1575.23,183.154 1577.29,181.967 1579.36,191.295 1581.42,186.885 1583.48,195.026 1585.54,217.075 1587.61,204.694 1589.67,189.599 \n",
       "  1591.73,219.449 1593.8,188.751 1595.86,207.068 1597.92,167.72 1599.98,190.447 1602.05,209.443 1604.11,210.291 1606.17,186.037 1608.23,214.87 1610.3,176.709 \n",
       "  1612.36,191.804 1614.42,175.861 1616.48,180.101 1618.55,177.727 1620.61,188.581 1622.67,206.729 1624.73,136.512 1626.8,211.308 1628.86,180.949 1630.92,142.788 \n",
       "  1632.98,175.182 1635.05,224.198 1637.11,196.553 1639.17,176.709 1641.23,203.507 1643.3,195.535 1645.36,157.035 1647.42,174.334 1649.48,158.052 1651.55,146.858 \n",
       "  1653.61,176.2 1655.67,177.048 1657.73,200.793 1659.8,212.156 1661.86,189.599 1663.92,196.383 1665.98,170.434 1668.05,142.957 1670.11,134.647 1672.17,142.109 \n",
       "  1674.24,247.434 1676.3,146.35 1678.36,195.705 1680.42,166.363 1682.49,171.451 1684.55,155.847 1686.61,190.956 1688.67,219.11 1690.74,177.557 1692.8,147.028 \n",
       "  1694.86,167.55 1696.92,175.861 1698.99,165.006 1701.05,159.579 1703.11,150.929 1705.17,152.964 1707.24,178.066 1709.3,156.526 1711.36,145.671 1713.42,152.455 \n",
       "  1715.49,154.999 1717.55,154.151 1719.61,197.231 1721.67,188.751 1723.74,181.627 1725.8,192.821 1727.86,135.156 1729.92,192.991 1731.99,146.01 1734.05,151.268 \n",
       "  1736.11,133.799 1738.17,156.695 1740.24,183.663 1742.3,219.28 1744.36,164.667 1746.42,238.276 1748.49,178.235 1750.55,180.949 1752.61,190.617 1754.67,193.5 \n",
       "  1756.74,190.956 1758.8,169.755 1760.86,195.874 1762.93,192.652 1764.99,180.271 1767.05,228.608 1769.11,139.057 1771.18,157.713 1773.24,181.288 1775.3,175.013 \n",
       "  1777.36,171.282 1779.43,201.641 1781.49,135.495 1783.55,179.083 1785.61,174.334 1787.68,160.427 1789.74,159.24 1791.8,189.09 1793.86,183.154 1795.93,204.524 \n",
       "  1797.99,191.804 1800.05,142.957 1802.11,148.724 1804.18,154.151 1806.24,182.815 1808.3,170.434 1810.36,165.515 1812.43,155.678 1814.49,156.526 1816.55,144.145 \n",
       "  1818.61,204.863 1820.68,167.381 1822.74,170.264 1824.8,177.557 1826.86,143.805 1828.93,129.219 1830.99,158.731 1833.05,174.674 1835.11,171.96 1837.18,130.237 \n",
       "  1839.24,175.013 1841.3,192.482 1843.37,156.865 1845.43,183.324 1847.49,185.528 1849.55,147.876 1851.62,156.017 1853.68,204.015 1855.74,142.957 1857.8,179.423 \n",
       "  1859.87,158.561 1861.93,139.396 1863.99,178.235 1866.05,151.777 1868.12,186.546 1870.18,174.334 1872.24,225.555 1874.3,156.187 1876.37,180.779 1878.43,158.731 \n",
       "  1880.49,174.504 1882.55,147.367 1884.62,143.636 1886.68,137.021 1888.74,142.618 1890.8,139.565 1892.87,165.176 1894.93,152.286 1896.99,192.821 1899.05,188.242 \n",
       "  1901.12,189.938 1903.18,132.272 1905.24,210.291 1907.3,145.332 1909.37,146.18 1911.43,179.592 1913.49,142.449 1915.55,160.088 1917.62,135.664 1919.68,151.438 \n",
       "  1921.74,176.2 1923.8,124.47 1925.87,176.37 1927.93,155.508 1929.99,166.363 1932.06,139.565 1934.12,149.063 1936.18,169.925 1938.24,154.999 1940.31,216.057 \n",
       "  1942.37,138.378 1944.43,150.759 1946.49,184.002 1948.56,155.678 1950.62,157.035 1952.68,200.793 1954.74,152.455 1956.81,136.173 1958.87,135.834 1960.93,181.458 \n",
       "  1962.99,182.815 1965.06,161.953 1967.12,117.856 1969.18,191.125 1971.24,156.187 1973.31,185.528 1975.37,197.231 1977.43,163.649 1979.49,170.603 1981.56,183.324 \n",
       "  1983.62,160.596 1985.68,166.024 1987.74,168.737 1989.81,130.746 1991.87,145.162 1993.93,154.321 1995.99,141.261 1998.06,120.739 2000.12,166.533 2002.18,154.321 \n",
       "  2004.24,143.297 2006.31,163.819 2008.37,169.585 2010.43,197.231 2012.5,134.138 2014.56,162.971 2016.62,186.376 2018.68,125.319 2020.75,120.739 2022.81,123.622 \n",
       "  2024.87,146.519 2026.93,158.222 2029,142.279 2031.06,158.561 2033.12,153.982 2035.18,166.024 2037.25,161.444 2039.31,158.392 2041.37,150.42 2043.43,123.283 \n",
       "  2045.5,147.028 2047.56,145.332 2049.62,163.48 2051.68,153.473 2053.75,183.663 2055.81,154.66 2057.87,148.385 2059.93,164.158 2062,174.504 2064.06,122.605 \n",
       "  2066.12,158.9 2068.18,143.127 2070.25,160.088 2072.31,121.926 2074.37,140.583 2076.43,140.413 2078.5,156.187 2080.56,110.902 2082.62,163.31 2084.68,155.678 \n",
       "  2086.75,163.649 2088.81,151.438 2090.87,148.554 2092.94,138.039 2095,146.01 2097.06,120.061 2099.12,177.048 2101.19,161.275 2103.25,149.233 2105.31,149.911 \n",
       "  2107.37,131.933 2109.44,109.206 2111.5,148.724 2113.56,150.081 2115.62,129.05 2117.69,169.585 2119.75,145.162 2121.81,165.006 2123.87,158.052 2125.94,148.385 \n",
       "  2128,115.651 2130.06,143.805 2132.12,167.381 2134.19,152.116 2136.25,140.753 2138.31,158.561 2140.37,136.004 2142.44,176.879 2144.5,155.678 2146.56,153.982 \n",
       "  2148.62,171.79 2150.69,136.004 2152.75,128.711 2154.81,117.517 2156.87,119.891 2158.94,158.731 2161,119.891 2163.06,151.438 2165.12,88.0053 2167.19,200.114 \n",
       "  2169.25,139.905 2171.31,161.275 2173.37,145.502 2175.44,140.583 2177.5,137.191 2179.56,123.453 2181.63,153.982 2183.69,138.209 2185.75,167.72 2187.81,164.497 \n",
       "  2189.88,133.799 2191.94,157.374 2194,155.847 2196.06,139.735 2198.13,176.03 2200.19,146.689 2202.25,143.805 2204.31,159.24 2206.38,145.502 2208.44,134.138 \n",
       "  2210.5,131.594 2212.56,170.603 2214.63,140.753 2216.69,137.53 2218.75,136.512 2220.81,165.345 2222.88,132.103 2224.94,123.792 2227,143.975 2229.06,143.466 \n",
       "  2231.13,130.407 2233.19,157.204 2235.25,128.032 2237.31,156.356 2239.38,131.594 2241.44,132.951 2243.5,116.838 2245.56,146.35 2247.63,136.004 2249.69,126.845 \n",
       "  2251.75,154.83 2253.81,159.07 2255.88,176.37 2257.94,112.768 2260,187.394 2262.07,89.871 2264.13,125.149 2266.19,169.755 2268.25,152.795 2270.32,156.017 \n",
       "  2272.38,118.874 2274.44,137.021 2276.5,125.319 2278.57,117.856 2280.63,111.411 2282.69,151.099 2284.75,139.905 2286.82,135.834 2288.88,139.057 2290.94,134.308 \n",
       "  \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8802)\" style=\"stroke:#e26f46; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  230.442,1228.43 232.504,1315.95 234.567,1283.89 236.63,1246.24 238.692,1278.97 240.755,1257.94 242.817,1248.28 244.88,1119.88 246.942,1097.84 249.005,1241.83 \n",
       "  251.067,1298.14 253.13,1201.46 255.193,1161.61 257.255,1292.71 259.318,1198.07 261.38,1137.52 263.443,1252.52 265.505,1236.4 267.568,1171.27 269.631,1269.14 \n",
       "  271.693,1132.1 273.756,1105.98 275.818,1150.58 277.881,1207.06 279.943,1281.35 282.006,1151.6 284.068,1175.51 286.131,1062.05 288.194,1107.84 290.256,1207.4 \n",
       "  292.319,1173.65 294.381,1022.87 296.444,1155.16 298.506,1146.85 300.569,1010.66 302.632,1194.68 304.694,1120.05 306.757,1059.5 308.819,995.903 310.882,1023.72 \n",
       "  312.944,1115.64 315.007,1119.54 317.069,999.125 319.132,1160.42 321.195,1027.11 323.257,1016.93 325.32,950.279 327.382,859.37 329.445,900.924 331.507,1112.76 \n",
       "  333.57,981.317 335.633,1066.29 337.695,950.279 339.758,880.401 341.82,947.565 343.883,1047.8 345.945,946.887 348.008,973.175 350.07,848.176 352.133,1021.68 \n",
       "  354.196,1154.65 356.258,1013.03 358.321,1081.72 360.383,994.715 362.446,906.69 364.508,938.067 366.571,1014.9 368.633,1070.36 370.696,973.854 372.759,1164.15 \n",
       "  374.821,979.79 376.884,920.259 378.946,1093.76 381.009,884.811 383.071,1005.74 385.134,893.461 387.197,1054.42 389.259,1009.64 391.322,855.809 393.384,1020.16 \n",
       "  395.447,881.08 397.509,771.515 399.572,964.017 401.634,1111.4 403.697,929.756 405.76,840.374 407.822,1034.91 409.885,813.238 411.947,881.589 414.01,800.517 \n",
       "  416.072,1052.04 418.135,936.201 420.198,1027.96 422.26,935.523 424.323,908.047 426.385,921.615 428.448,797.295 430.51,773.38 432.573,759.303 434.635,939.594 \n",
       "  436.698,806.962 438.761,832.233 440.823,747.261 442.886,1060.18 444.948,976.398 447.011,787.458 449.073,748.957 451.136,938.576 453.199,1030.16 455.261,715.884 \n",
       "  457.324,772.872 459.386,863.61 461.449,804.249 463.511,984.369 465.574,860.897 467.636,886.677 469.699,910.93 471.762,744.717 473.824,910.082 475.887,803.231 \n",
       "  477.949,760.321 480.012,925.686 482.074,876.161 484.137,840.714 486.2,667.547 488.262,864.967 490.325,780.334 492.387,800.687 494.45,749.127 496.512,802.213 \n",
       "  498.575,963.338 500.637,858.522 502.7,728.435 504.763,793.394 506.825,745.735 508.888,808.828 510.95,833.59 513.013,829.52 515.075,880.571 517.138,772.702 \n",
       "  519.2,696.38 521.263,753.706 523.326,779.317 525.388,907.708 527.451,711.644 529.513,722.668 531.576,762.186 533.638,792.207 535.701,703.333 537.764,748.109 \n",
       "  539.826,818.495 541.889,804.927 543.951,833.76 546.014,662.967 548.076,732.166 550.139,641.088 552.201,701.977 554.264,723.008 556.327,792.715 558.389,926.025 \n",
       "  560.452,752.689 562.514,734.032 564.577,700.45 566.639,703.503 568.702,785.592 570.765,700.45 572.827,752.18 574.89,715.884 576.952,715.036 579.015,773.38 \n",
       "  581.077,746.583 583.14,706.726 585.202,757.268 587.265,697.736 589.328,705.538 591.39,606.998 593.453,723.008 595.515,697.736 597.578,734.88 599.64,833.421 \n",
       "  601.703,666.699 603.766,701.129 605.828,705.877 607.891,639.223 609.953,669.582 612.016,726.739 614.078,732.506 616.141,631.93 618.203,635.661 620.266,711.305 \n",
       "  622.329,663.646 624.391,533.728 626.454,714.867 628.516,660.254 630.579,760.49 632.641,726.739 634.704,724.195 636.767,759.812 638.829,678.741 640.892,709.27 \n",
       "  642.954,611.577 645.017,511.34 647.079,555.946 649.142,625.145 651.204,697.058 653.267,693.666 655.33,680.776 657.392,736.237 659.455,565.275 661.517,700.45 \n",
       "  663.58,725.382 665.642,654.487 667.705,560.017 669.767,738.102 671.83,708.93 673.893,736.067 675.955,648.042 678.018,586.475 680.08,535.424 682.143,463.342 \n",
       "  684.205,547.466 686.268,585.967 688.331,653.469 690.393,584.61 692.456,665.172 694.518,538.647 696.581,625.145 698.643,743.699 700.706,559.508 702.768,493.532 \n",
       "  704.831,563.409 706.894,671.617 708.956,634.813 711.019,605.641 713.081,564.596 715.144,597.5 717.206,528.979 719.269,795.09 721.332,740.477 723.394,503.199 \n",
       "  725.457,595.973 727.519,519.312 729.582,536.272 731.644,604.114 733.707,514.902 735.769,533.05 737.832,558.49 739.895,595.973 741.957,713.679 744.02,616.156 \n",
       "  746.082,646.685 748.145,545.6 750.207,546.957 752.27,551.028 754.333,538.986 756.395,505.234 758.458,655.844 760.52,555.607 762.583,606.658 764.645,558.999 \n",
       "  766.708,554.42 768.77,561.883 770.833,541.021 772.896,531.523 774.958,518.294 777.021,556.794 779.083,590.885 781.146,626.672 783.208,621.923 785.271,638.035 \n",
       "  787.334,555.098 789.396,525.078 791.459,521.856 793.521,584.101 795.584,517.446 797.646,517.785 799.709,528.64 801.771,557.303 803.834,477.08 805.897,554.759 \n",
       "  807.959,574.264 810.022,487.935 812.084,451.13 814.147,577.825 816.209,511.001 818.272,544.583 820.334,607.846 822.397,551.876 824.46,567.649 826.522,451.47 \n",
       "  828.585,522.365 830.647,578.843 832.71,548.653 834.772,466.395 836.835,515.072 838.898,587.323 840.96,521.177 843.023,757.437 845.085,574.773 847.148,531.354 \n",
       "  849.21,557.134 851.273,482.507 853.335,532.88 855.398,446.212 857.461,660.254 859.523,530.166 861.586,553.233 863.648,431.965 865.711,364.462 867.773,470.465 \n",
       "  869.836,530.506 871.899,630.742 873.961,468.6 876.024,479.285 878.086,534.067 880.149,602.927 882.211,544.922 884.274,542.378 886.336,449.434 888.399,511.171 \n",
       "  890.462,542.378 892.524,475.045 894.587,490.648 896.649,435.018 898.712,527.453 900.774,624.297 902.837,416.192 904.9,546.618 906.962,477.419 909.025,498.789 \n",
       "  911.087,541.191 913.15,491.327 915.212,481.829 917.275,538.986 919.337,470.635 921.4,556.794 923.463,472.501 925.525,466.225 927.588,503.878 929.65,523.382 \n",
       "  931.713,472.501 933.775,502.86 935.838,497.093 937.901,493.192 939.963,481.32 942.026,483.525 944.088,546.788 946.151,507.778 948.213,470.296 950.276,528.131 \n",
       "  952.338,505.234 954.401,552.045 956.464,475.045 958.526,446.89 960.589,489.461 962.651,457.575 964.714,512.019 966.776,498.789 968.839,475.045 970.901,430.947 \n",
       "  972.964,580.878 975.027,385.832 977.089,414.326 979.152,463.851 981.214,448.586 983.277,518.633 985.339,510.153 987.402,503.878 989.465,470.974 991.527,483.864 \n",
       "  993.59,444.176 995.652,467.243 997.715,413.139 999.777,447.569 1001.84,508.118 1003.9,439.428 1005.97,481.32 1008.03,534.576 1010.09,458.593 1012.15,487.426 \n",
       "  1014.22,526.435 1016.28,525.757 1018.34,447.569 1020.4,472.501 1022.47,401.097 1024.53,477.08 1026.59,493.023 1028.65,531.354 1030.72,482.847 1032.78,461.815 \n",
       "  1034.84,526.944 1036.9,505.234 1038.97,473.518 1041.03,492.344 1043.09,479.963 1045.15,447.229 1047.22,459.271 1049.28,444.176 1051.34,368.872 1053.4,386.511 \n",
       "  1055.47,542.208 1057.53,471.653 1059.59,486.408 1061.65,495.737 1063.72,386.171 1065.78,442.65 1067.84,430.947 1069.9,418.227 1071.97,483.525 1074.03,437.562 \n",
       "  1076.09,526.944 1078.15,467.752 1080.22,469.278 1082.28,394.991 1084.34,492.514 1086.4,491.327 1088.47,442.65 1090.53,365.31 1092.59,385.154 1094.66,417.209 \n",
       "  1096.72,361.748 1098.78,510.323 1100.84,377.182 1102.91,431.965 1104.97,512.867 1107.03,461.985 1109.09,466.225 1111.16,406.185 1113.22,374.299 1115.28,503.369 \n",
       "  1117.34,449.773 1119.41,459.611 1121.47,399.231 1123.53,459.78 1125.59,437.223 1127.66,403.81 1129.72,426.198 1131.78,387.698 1133.84,399.74 1135.91,433.491 \n",
       "  1137.97,427.386 1140.03,439.936 1142.09,406.355 1144.16,454.014 1146.22,395.5 1148.28,387.359 1150.34,417.04 1152.41,371.585 1154.47,333.763 1156.53,460.628 \n",
       "  1158.59,395.33 1160.66,386.511 1162.72,472.84 1164.78,375.486 1166.84,381.253 1168.91,379.218 1170.97,439.597 1173.03,372.942 1175.1,432.813 1177.16,391.599 \n",
       "  1179.22,402.454 1181.28,401.606 1183.35,400.927 1185.41,408.39 1187.47,423.485 1189.53,413.817 1191.6,427.216 1193.66,439.767 1195.72,414.835 1197.78,480.472 \n",
       "  1199.85,349.367 1201.91,485.899 1203.97,361.239 1206.03,356.491 1208.1,380.914 1210.16,398.722 1212.22,402.454 1214.28,401.606 1216.35,341.904 1218.41,413.308 \n",
       "  1220.47,387.868 1222.53,385.832 1224.6,392.447 1226.66,408.729 1228.72,407.372 1230.78,427.894 1232.85,416.531 1234.91,344.957 1236.97,427.894 1239.03,450.961 \n",
       "  1241.1,371.077 1243.16,339.869 1245.22,400.588 1247.28,390.581 1249.35,330.202 1251.41,415.513 1253.47,300.521 1255.53,361.239 1257.6,407.542 1259.66,314.428 \n",
       "  1261.72,347.841 1263.79,348.519 1265.85,381.592 1267.91,386.85 1269.97,424.502 1272.04,413.648 1274.1,356.66 1276.16,420.771 1278.22,378.2 1280.29,394.652 \n",
       "  1282.35,320.704 1284.41,418.057 1286.47,386.341 1288.54,408.051 1290.6,420.941 1292.66,433.152 1294.72,375.147 1296.79,384.645 1298.85,356.83 1300.91,344.957 \n",
       "  1302.97,342.753 1305.04,371.246 1307.1,381.931 1309.16,375.486 1311.22,440.954 1313.29,419.414 1315.35,396.178 1317.41,446.721 1319.47,280.677 1321.54,382.101 \n",
       "  1323.6,395.5 1325.66,325.114 1327.72,316.973 1329.79,358.017 1331.85,336.477 1333.91,348.519 1335.97,323.248 1338.04,368.872 1340.1,334.442 1342.16,391.938 \n",
       "  1344.23,353.777 1346.29,416.361 1348.35,369.381 1350.41,411.782 1352.48,368.872 1354.54,456.218 1356.6,333.763 1358.66,342.753 1360.73,341.565 1362.79,327.827 \n",
       "  1364.85,335.12 1366.91,348.349 1368.98,410.934 1371.04,390.581 1373.1,274.91 1375.16,399.401 1377.23,321.043 1379.29,365.819 1381.35,370.398 1383.41,392.956 \n",
       "  1385.48,368.872 1387.54,297.807 1389.6,403.98 1391.66,348.689 1393.73,415.852 1395.79,409.407 1397.85,335.12 1399.91,364.462 1401.98,319.517 1404.04,314.768 \n",
       "  1406.1,382.61 1408.16,330.202 1410.23,321.721 1412.29,339.869 1414.35,317.99 1416.41,352.42 1418.48,341.904 1420.54,312.224 1422.6,389.224 1424.67,305.779 \n",
       "  1426.73,383.458 1428.79,417.548 1430.85,368.702 1432.92,302.047 1434.98,381.931 1437.04,400.758 1439.1,402.793 1441.17,377.861 1443.23,316.124 1445.29,408.051 \n",
       "  1447.35,334.272 1449.42,316.124 1451.48,371.585 1453.54,356.83 1455.6,348.349 1457.67,312.224 1459.73,356.66 1461.79,349.537 1463.85,357.678 1465.92,355.473 \n",
       "  1467.98,354.455 1470.04,340.378 1472.1,363.953 1474.17,354.794 1476.23,310.528 1478.29,389.733 1480.35,296.281 1482.42,404.15 1484.48,402.623 1486.54,321.382 \n",
       "  1488.6,342.413 1490.67,336.477 1492.73,367.345 1494.79,412.46 1496.85,300.012 1498.92,384.136 1500.98,379.048 1503.04,297.129 1505.1,349.028 1507.17,288.479 \n",
       "  1509.23,378.878 1511.29,320.704 1513.36,352.42 1515.42,301.199 1517.48,296.281 1519.54,374.978 1521.61,367.684 1523.67,330.541 1525.73,342.244 1527.79,312.054 \n",
       "  1529.86,321.213 1531.92,338.173 1533.98,359.713 1536.04,347.671 1538.11,267.787 1540.17,284.917 1542.23,325.792 1544.29,309.849 1546.36,318.499 1548.42,283.221 \n",
       "  1550.48,316.803 1552.54,283.221 1554.61,317.142 1556.67,345.466 1558.73,284.917 1560.79,374.299 1562.86,356.491 1564.92,389.394 1566.98,380.914 1569.04,319.686 \n",
       "  1571.11,280.507 1573.17,310.358 1575.23,288.648 1577.29,297.638 1579.36,344.279 1581.42,331.728 1583.48,348.858 1585.54,369.211 1587.61,284.578 1589.67,277.794 \n",
       "  1591.73,349.367 1593.8,367.176 1595.86,308.662 1597.92,271.349 1599.98,295.093 1602.05,308.492 1604.11,279.829 1606.17,370.568 1608.23,268.974 1610.3,257.441 \n",
       "  1612.36,296.45 1614.42,360.731 1616.48,267.957 1618.55,390.412 1620.61,292.041 1622.67,410.934 1624.73,357.169 1626.8,274.232 1628.86,328.845 1630.92,323.757 \n",
       "  1632.98,326.47 1635.05,300.86 1637.11,330.371 1639.17,299.673 1641.23,327.488 1643.3,309.171 1645.36,334.951 1647.42,313.411 1649.48,345.466 1651.55,339.869 \n",
       "  1653.61,307.305 1655.67,313.58 1657.73,346.314 1659.8,279.829 1661.86,274.232 1663.92,316.633 1665.98,339.869 1668.05,307.644 1670.11,300.521 1672.17,317.142 \n",
       "  1674.24,301.199 1676.3,333.594 1678.36,308.662 1680.42,328.675 1682.49,309.34 1684.55,355.812 1686.61,285.256 1688.67,301.369 1690.74,333.763 1692.8,274.062 \n",
       "  1694.86,301.708 1696.92,291.532 1698.99,276.267 1701.05,326.47 1703.11,308.492 1705.17,303.913 1707.24,305.27 1709.3,330.711 1711.36,331.898 1713.42,258.628 \n",
       "  1715.49,294.415 1717.55,234.714 1719.61,284.069 1721.67,267.787 1723.74,271.179 1725.8,303.404 1727.86,226.064 1729.92,292.719 1731.99,300.012 1734.05,295.263 \n",
       "  1736.11,295.772 1738.17,300.182 1740.24,299.842 1742.3,252.522 1744.36,248.113 1746.42,341.565 1748.49,286.783 1750.55,336.138 1752.61,278.472 1754.67,301.369 \n",
       "  1756.74,286.613 1758.8,272.366 1760.86,259.816 1762.93,323.587 1764.99,291.362 1767.05,338.343 1769.11,290.853 1771.18,316.803 1773.24,280.507 1775.3,330.371 \n",
       "  1777.36,278.811 1779.43,304.422 1781.49,266.43 1783.55,314.768 1785.61,285.256 1787.68,280.847 1789.74,347.671 1791.8,234.375 1793.86,331.728 1795.93,292.21 \n",
       "  1797.99,229.626 1800.05,296.281 1802.11,217.245 1804.18,308.323 1806.24,238.784 1808.3,301.708 1810.36,306.457 1812.43,228.099 1814.49,271.688 1816.55,316.294 \n",
       "  1818.61,249.639 1820.68,243.873 1822.74,284.747 1824.8,296.62 1826.86,273.554 1828.93,238.784 1830.99,255.406 1833.05,229.965 1835.11,273.384 1837.18,259.307 \n",
       "  1839.24,243.703 1841.3,306.796 1843.37,352.929 1845.43,304.931 1847.49,272.536 1849.55,269.653 1851.62,246.417 1853.68,329.014 1855.74,288.648 1857.8,283.051 \n",
       "  1859.87,276.606 1861.93,273.893 1863.99,257.611 1866.05,263.377 1868.12,238.276 1870.18,263.547 1872.24,231.152 1874.3,340.717 1876.37,280.847 1878.43,245.569 \n",
       "  1880.49,263.547 1882.55,259.476 1884.62,279.999 1886.68,260.664 1888.74,228.439 1890.8,278.133 1892.87,275.419 1894.93,256.423 1896.99,277.624 1899.05,255.575 \n",
       "  1901.12,248.791 1903.18,256.593 1905.24,224.707 1907.3,292.38 1909.37,284.239 1911.43,269.992 1913.49,219.11 1915.55,199.097 1917.62,238.954 1919.68,272.027 \n",
       "  1921.74,297.129 1923.8,299.673 1925.87,267.448 1927.93,288.648 1929.99,264.564 1932.06,246.077 1934.12,330.032 1936.18,221.654 1938.24,271.688 1940.31,268.635 \n",
       "  1942.37,270.331 1944.43,236.919 1946.49,239.802 1948.56,358.695 1950.62,242.685 1952.68,243.873 1954.74,296.62 1956.81,251.335 1958.87,252.014 1960.93,271.009 \n",
       "  1962.99,303.743 1965.06,253.879 1967.12,249.978 1969.18,270.501 1971.24,257.78 1973.31,245.908 1975.37,229.456 1977.43,304.591 1979.49,268.126 1981.56,267.787 \n",
       "  1983.62,236.41 1985.68,303.404 1987.74,321.891 1989.81,314.768 1991.87,276.098 1993.93,242.346 1995.99,239.972 1998.06,324.944 2000.12,259.476 2002.18,207.916 \n",
       "  2004.24,227.76 2006.31,269.483 2008.37,206.559 2010.43,229.795 2012.5,250.657 2014.56,296.789 2016.62,247.265 2018.68,236.071 2020.75,217.753 2022.81,229.117 \n",
       "  2024.87,288.479 2026.93,248.282 2029,256.254 2031.06,277.115 2033.12,246.756 2035.18,253.54 2037.25,243.364 2039.31,291.193 2041.37,257.611 2043.43,244.721 \n",
       "  2045.5,260.664 2047.56,196.553 2049.62,255.406 2051.68,243.364 2053.75,251.505 2055.81,243.873 2057.87,220.637 2059.93,244.89 2062,252.183 2064.06,238.615 \n",
       "  2066.12,218.093 2068.18,209.104 2070.25,314.937 2072.31,241.498 2074.37,213.683 2076.43,198.418 2078.5,216.736 2080.56,275.589 2082.62,297.468 2084.68,225.555 \n",
       "  2086.75,204.524 2088.81,248.791 2090.87,230.474 2092.94,266.769 2095,240.481 2097.06,237.428 2099.12,228.947 2101.19,269.144 2103.25,242.007 2105.31,239.463 \n",
       "  2107.37,265.752 2109.44,307.814 2111.5,231.152 2113.56,238.276 2115.62,259.476 2117.69,201.132 2119.75,266.769 2121.81,290.005 2123.87,260.324 2125.94,258.459 \n",
       "  2128,261.512 2130.06,229.795 2132.12,242.007 2134.19,220.297 2136.25,302.217 2138.31,208.256 2140.37,219.619 2142.44,250.148 2144.5,228.439 2146.56,232 \n",
       "  2148.62,261.681 2150.69,266.43 2152.75,240.989 2154.81,241.837 2156.87,215.718 2158.94,251.674 2161,296.62 2163.06,208.425 2165.12,269.313 2167.19,253.371 \n",
       "  2169.25,275.589 2171.31,176.37 2173.37,224.029 2175.44,280.338 2177.5,254.219 2179.56,228.269 2181.63,267.448 2183.69,239.124 2185.75,234.714 2187.81,232.679 \n",
       "  2189.88,289.496 2191.94,237.428 2194,250.487 2196.06,243.194 2198.13,225.216 2200.19,228.099 2202.25,222.333 2204.31,221.146 2206.38,255.745 2208.44,198.249 \n",
       "  2210.5,232.339 2212.56,304.252 2214.63,210.46 2216.69,218.941 2218.75,249.3 2220.81,243.533 2222.88,208.934 2224.94,233.866 2227,233.357 2229.06,226.573 \n",
       "  2231.13,194.009 2233.19,274.741 2235.25,276.267 2237.31,238.954 2239.38,236.071 2241.44,234.544 2243.5,252.862 2245.56,256.423 2247.63,222.502 2249.69,290.005 \n",
       "  2251.75,270.331 2253.81,205.542 2255.88,207.916 2257.94,239.972 2260,200.454 2262.07,220.976 2264.13,244.042 2266.19,205.881 2268.25,240.82 2270.32,223.011 \n",
       "  2272.38,241.837 2274.44,226.403 2276.5,195.196 2278.57,254.049 2280.63,205.881 2282.69,304.761 2284.75,246.247 2286.82,221.485 2288.88,249.13 2290.94,228.608 \n",
       "  \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip8800)\" d=\"\n",
       "M1737.18 1343.47 L2280.76 1343.47 L2280.76 1162.03 L1737.18 1162.03  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip8800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1737.18,1343.47 2280.76,1343.47 2280.76,1162.03 1737.18,1162.03 1737.18,1343.47 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip8800)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1761.18,1222.51 1905.18,1222.51 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip8800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1929.18, 1240.01)\" x=\"1929.18\" y=\"1240.01\">alpha = 2^-13</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip8800)\" style=\"stroke:#e26f46; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1761.18,1282.99 1905.18,1282.99 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip8800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1929.18, 1300.49)\" x=\"1929.18\" y=\"1300.49\">alpha = 2^-14</text>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = plot(legend=:bottomright)\n",
    "for x in [-13, -14]  # for -12, it seems not that easy to be convergent in short time\n",
    "    plot!(p, mean(run_once_RL(2. ^ x) for _ in 1:100), label=\"alpha = 2^$x\")\n",
    "end\n",
    "p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise\n",
    "\n",
    "Now can you implement a `ReinforceWithBaselinePolicy` by your self to reproduce the figure 13.2?"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.3.0",
   "language": "julia",
   "name": "julia-1.3"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.3.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
